***********************************************************************
NOTE: Due to disk limitations in working dir, switch work directory to 
      home dir: sas -work /mnt/research_disk_1/newhome/meyer1 poverty_trends_health60_16 &

9/15/18: ADDED 2017 DATA

9/6/17: ADDED 2015 & 2016 DATA

11/5/15: ADDED 2014 DATA

10/20/14: ADDED 2013 DATA

9/12/13: ADDED 2012 DATA  

12/20/12: ADDED 2011 DATA 

10/2/12: CHANGED THE MAIN PRICE INDEX TO ADJUSTED RS, CHANGED GAP TO BE  
				 PER PERSON RATHER THAN PER FAMILY, PRICES_1980 NOW HAS ADJUSTED_RS
				 PRE 1977 USING 0.011.  MOVED AWAY FROM USING GUNZIP MACRO
				 ALL DOLLAR FIGS NOW IN 2010 $

11/3/11: ADDED 2010 DATA 

1/1/11: ADDED 2009 THROUGH THE END OF PROGRAM, FIXED CPI_U_RS WHICH WAS OFF 
	IN THIRD DECIMAL PLACE 06-08. 

11/3/10: ADDED 2008 THROUGH THE CREATION OF CONS_FINAL DATA SET, WHICH
	 I NOW CALL CONS_FINAL60_0*.

1/15/10: EXTENDED REST OF PROGRAM THROUGH 2008 

12/20/09: ADDED 2008 THROUGH THE CREATION OF CONS_FINAL DATA SET. 

8/27/09: ADDED 2006 & 2007

1/9/09: ADDED 1960-61

6/4/08: ADDED CONS8 & CONS9, FOR INCOME POVERTY MEASURES.  ADDED PROC
	FREQ FOR FTYPES AND EMPLOYMENT CONFIGURATION

3/21/08: CHANGED CONS2 TO CONSUMPTION PLUS EDUCATION SPENDING ADDED
  	 MORE DEMOGRAPIC CHARACTERISTICS AND RAN DEMOG STUFF BY DEMOG	 GROUP

3/21/08: UPDATED TO REFLECT ADDITION OF 2005 DATA, TFLOW2 FOR 1980-81 NO
         LONGER INCLUDES IMPUTED VALUE OF RENT FOR THOSE IN PUBLIC OR
         SUBSIDIZED HOUSING.  DEFINE CONS5 AS CORE CONSUMPTION RATHER THAN
         CONSUMPTION PLUS MOOP

1/10/08: CREATED MUTUALLY EXCLUSIVE DEMOGRAPHIC GROUPS

12/30/07: ADDED 72-73

8/12/07: UPDATED TO REFLECT CHANGES MADE FOR ELDERLY POVERTY INCLUDING
ADDING HEALTH, VEHICLES, AND ANCHORING IN 1980

ALSO CHANGED SINGLE MOM MEANS TO BE FOR ALL PERSONS IN LONE SM FAMS

9/6/07: ADDED ADJUSTED CPI-U-RS AND RELATIVE POVERTY
***********************************************************************;
OPTIONS NOLABEL;

FILENAME outfile1 '~/jim/poverty/output/povall.dat';
FILENAME outfile2 '~/jim/poverty/output/pov65.dat';
FILENAME outfile3 '~/jim/poverty/output/demog_notbyftype.dat';
FILENAME outfile4 '~/jim/poverty/output/demog_ass_dist.dat';
FILENAME outfile5 '~/jim/poverty/output/medians.dat';
FILENAME outfile6 '~/jim/poverty/output/povgaps.dat';
FILENAME outfile7 '~/jim/poverty/output/demog.dat';
FILENAME outfile8 '~/jim/poverty/output/stata_se.dat';
FILENAME outfile9 '~/jim/poverty/output/demog_byftype.dat';
FILENAME outfil10 '~/jim/poverty/output/asset_byftype.dat';
FILENAME outfil11 '~/jim/poverty/output/shares.dat';
FILENAME outfil12 '~/jim/poverty/output/incpov.dat';
FILENAME outfil13 '~/jim/poverty/output/demog_inc.dat';
FILENAME outfil14 '~/jim/poverty/output/ftype_dist_by_yr.dat';
FILENAME outfil15 '~/jim/poverty/output/pov_by_dem.dat';
FILENAME outfil16 '~/jim/poverty/output/pov_by_dem2.dat';
FILENAME outfil17 '~/jim/poverty/output/demog_official.dat';
FILENAME outfil18 '~/jim/poverty/output/asset_notbyftype.dat';
FILENAME outfil20 '~/jim/poverty/output/off_poverty.dat';
FILENAME outfil21 '~/jim/poverty/output/pov_by_age.dat';

LIBNAME alldat '~/jim/data';


***********************************************************************
MERGE IN DEBT DATA
***********************************************************************;
PROC SORT DATA=alldat.cons_exp80_17_plus_veh_memb;
  BY qyear newid;
RUN;

* NOTE LEAVE AS 2013, WE DON'T UPDATE DEBTVAL AFTER 2013 BECAUSE THEY NOW HAVE THIS DATA IN THE FAMILY FILE;
* SEE CREDITX OTHLONX. IT APPEARS THESE ARE ONLY ASKED IN 5TH INTERVIEW. IT USED TO BE 2ND AND 5TH INTERVIEWS;
PROC SORT DATA=alldat.debt_val80_13;
  BY qyear newid; 
RUN;

DATA cons_exp80_17_plus_veh_memb;
   MERGE alldat.cons_exp80_17_plus_veh_memb(IN=in1) alldat.debt_val80_13;
   BY qyear newid; 
   IF in1;

lft5=floor(newid/10);
rt1=newid-lft5*10;


* NOTE, FLAG = A MEANS VALID NONRESPONSE, IE VALID ZER0;
IF (1132 LE qyear LE 1145) OR srv_year=115 AND desflg='00' THEN DO;
	IF creditx=. AND rt1=5 AND creditx_ = 'A' THEN debt1=0;
	   ELSE debt1=creditx;
	IF othlonx=. AND rt1=5 AND othlonx_ = 'A' THEN debt2=0;
	   ELSE debt2=othlonx;
	IF studntx=. AND rt1=5 AND studntx_ = 'A' THEN debt3=0;
	   ELSE debt3=studntx;
END;
IF (srv_year GE 116) OR srv_year=115 AND desflg='10' THEN DO;
	IF creditx=. AND rt1=4 AND creditx_ = 'A' THEN debt1=0;
	   ELSE debt1=creditx;
	IF othlonx=. AND rt1=4 AND othlonx_ = 'A' THEN debt2=0;
	   ELSE debt2=othlonx;
	IF studntx=. AND rt1=4 AND studntx_ = 'A' THEN debt3=0;
	   ELSE debt3=studntx;
END;

* NOTE IN OLDER YEARS, DEBT COMES FROM UCC CODES, NO WAY TO EXCLUDE INVALID NONRESPONSES SUCH AS REFUSALS;
IF (qyear LE 1131) THEN DO;
	IF tot_debt=. AND rt1 IN(2 5) THEN tot_debt2=0;
	   ELSE tot_debt2=tot_debt;
END;

IF (qyear GE 1132) THEN tot_debt2=SUM(OF debt1 debt2 debt3);

RUN;

***********************************************************************
CLEAN 72-73 DATA
***********************************************************************;
DATA pov7273;
  SET alldat.famstep7273(RENAME=(vflow_pred=n_tot_vflow1));

* "EXCLUDE" VARIABLE IS FOR INC NONRESPONDERS 1980-PRESENT;
exclude=0;

IF locat=8 THEN bls_urbn=2;
   ELSE IF 1 LE locat LE 7 THEN bls_urbn=1; 

core_cons=d_fdhome+d_rent+d_utility
		  +d_gas+d_othvehic-d_autofin+d_pubtrans
		  +n_tot_vflow1+hflow;

****************************************
Define a consistent dataset
****************************************;
IF tflow ne .;


***********************************************************************************************************
***********************************************************************************************************
CREATE DIFFERENT MEASURES OF INCOME AND CONSUMPTION

***********************************************************************************************************
***********************************************************************************************************;
***********************
OFFICIAL INCOME POVERTY
***********************;
* FIRST DEFINE FOOD STAMPS;
fstamp=-1*foodstamp;

IF fstamp=. OR fincbtax=. THEN official_pov=.; 
   ELSE IF (fincbtax-fstamp)<pov_cut THEN official_pov=1;
   ELSE official_pov=0; 

scale=((fam_size-perslt18+.7*perslt18)**.7);

***********************
CONSUMPTION MEASURES
***********************;

* adjusted total expenditures;
cons1=(totexp2)/scale/4;

* consumption, no gov't housing adj, no HI;
* cons2=(tflow)/scale/4;
* consumption including education spending;
* cons2=(tflow+d_educ)/scale/4;
* CONS2: NOW DEFINED AS WELL-MEASURED CONSUMPTION (STARTING IN 2016);
cons2=(core_cons-(d_othvehic-d_autofin)-d_pubtrans)/scale/4;

* main consumption; 
cons3=(tflow)/scale/4;

* unadjusted total expenditures;
cons4=(totexp)/scale/4;

* consumption plus MOOP; 
*cons5=(tflow+d_health)/scale/4;
* CORE CONSUMPTION;
cons5=(core_cons)/scale/4;

* consumption no HI; 
cons6=(tflow)/scale/4;

* non-housing consumption; 
cons7=(tflow-SUM(OF d_rent hflow))/scale/4;

* after tax income;
* cons9=(fincatax+nonincmx-fica)/scale;
* CONS9: NOW DEFINED AS WELL-MEASURED CONSUMPTION LESS FOOD (STARTING IN 2016);
cons9=(core_cons-(d_othvehic-d_autofin)-d_pubtrans-d_fdhome)/scale/4;


***********************
INCOME MEASURES
***********************;

* before tax income;
IF fstamp=. OR fincbtax=. THEN cons8=.; 
   ELSE cons8=(fincbtax-fstamp)/scale;



IF married=0 AND sex=2 AND perslt18 GT 0 AND fam_size GT 1 
	THEN wgt23=finlwt*fam_size; 
   ELSE wgt23=0; 

IF totexp2<pov_cut THEN pov1=finlwt;
   ELSE pov1=0;
IF tflow<pov_cut THEN pov2=finlwt; 
   ELSE pov2=0;    
IF tflow<pov_cut THEN pov3=finlwt;
   ELSE pov3=0; 
gap1=(max(0,pov_cut-MAX(0,totexp2)))*finlwt;
gap2=(max(0,pov_cut-MAX(0,tflow)))*finlwt;
gap3=(max(0,pov_cut-MAX(0,tflow)))*finlwt;

RUN;

***********************************************************************
CLEAN 80-PRESNET DATA
***********************************************************************;
DATA poverty;
  SET cons_exp80_17_plus_veh_memb;

****************************************
DEFINE CORE CONSUMPTION
FOOD AT HOME + RENT + UTILITIES + RENTAL EQUIVALENT
+ VEHICLE FLOWS LESS VEHIC PURCHASE + RENTAL EQIV 
FOR THOSE IN PUBLIC OR SUBSIDIZED HOUSING.
NOTE:  TFLOW2-TFLOW IS RENT EQUIV
FOR THOSE IN PUBLIC OR SUBSIDIZED HOUSING.
****************************************;
* NOTE: THE DIFFERENCE BETWEEN foodcq2 AND foodcq IS THE AMT FOOD AT HOME IS ADJUSTED FOR 84-87.  SEE FAMSTEP80_00.SAS;
IF 831 LE qyear LE 874 THEN fdhomepq2=fdhomepq+(foodpq2-foodpq);
   ELSE fdhomepq2=fdhomepq;
IF 831 LE qyear LE 874 THEN fdhomecq2=fdhomecq+(foodcq2-foodcq);
   ELSE fdhomecq2=fdhomecq;

core_cons=fdhomepq2+fdhomecq2
          +rendwepq+rendwecq
          +utilpq+utilcq 
          +transpq+transcq
          +hflow
          +n_tot_vflow1
          -SUM(OF vehfinpq vehfincq
		  vehicpq vehiccq 
		  cartknpq cartkncq cartkupq cartkucq caropq carocq)
          +tflow2-tflow;

****************************************
RESTRICT TO OBS WITH MEMBER FILE MATCH
****************************************;
IF asm_dum1 NE .;

****************************************
CREATE AGE INDICATORS FROM MEMB FILE
****************************************;
pers6569=SUM(OF asm_dum1-asm_dum4);
pers7079=SUM(OF asm_dum5-asm_dum12);
pers80=SUM(OF asm_dum13-asm_dum16);
pers6574=SUM(OF asm_dum1-asm_dum8);
pers75=SUM(OF asm_dum9-asm_dum16); 

menot64=SUM(OF asm_dum1-asm_dum2 asm_dum5-asm_dum6 asm_dum9-asm_dum10 asm_dum13-asm_dum14);
men6569=SUM(OF asm_dum1-asm_dum2);
men7079=SUM(OF asm_dum5-asm_dum6 asm_dum9-asm_dum10);
men80=SUM(OF asm_dum13-asm_dum14);
men6574=SUM(OF asm_dum1-asm_dum2 asm_dum5-asm_dum6);
men75=SUM(OF asm_dum9-asm_dum10 asm_dum13-asm_dum14);

womenot64=SUM(OF asm_dum3-asm_dum4 asm_dum7-asm_dum8 asm_dum11-asm_dum12 asm_dum15-asm_dum16);
women6569=SUM(OF asm_dum3-asm_dum4);
women7079=SUM(OF asm_dum7-asm_dum8 asm_dum11-asm_dum12);
women80=SUM(OF asm_dum15-asm_dum16);
women6574=SUM(OF asm_dum3-asm_dum4 asm_dum7-asm_dum8);                                        
women75=SUM(OF asm_dum11-asm_dum12 asm_dum15-asm_dum16);                                      

marriedot64=SUM(OF asm_dum1 asm_dum3 asm_dum5 asm_dum7 asm_dum9 asm_dum11 asm_dum13 asm_dum15);
singot64=SUM(OF asm_dum2 asm_dum4 asm_dum6 asm_dum8 asm_dum10 asm_dum12 asm_dum14 asm_dum16);

ARRAY tmp(1:19)
          pers6569 pers7079 pers80 menot64 men6569 men7079 men80 womenot64 
          women6569 women7079 women80 marriedot64 singot64 pers6574 pers75 
	  men6574 men75 women6574 women75;
ARRAY wgt(1:19);
DO t=1 TO 19; 
   wgt(t)=finlwt*tmp(t);
END;


****************************************
CREATE KINK FOR HEALTH INSURANCE
****************************************;
pc_kink=(MIN(.333*totexp2,pc_total_insurance*fam_size/4))/fam_size;

****************************************
GET RID OF 85:5 DATA
****************************************;
IF qyear NE 855;

********************************************
GET RID OF DATA IN QTRS WHERE ONLY URBAN OBS
********************************************;
IF qyear NE 821;
IF qyear NE 815;
IF qyear NE 814;

*******************************************
Define a consistent dataset
*******************************************;
   IF tflow ne . AND n_tot_vflow1 NE . AND pc_total_insurance NE .;




***********************************************************************************************************
***********************************************************************************************************
CREATE DIFFERENT MEASURES OF INCOME AND CONSUMPTION

***********************************************************************************************************
***********************************************************************************************************;
***********************
OFFICIAL INCOME POVERTY
***********************;
* FIRST DEFINE FOOD STAMPS;
IF (801 LE qyear LE 815) OR (841 LE qyear LE 1011) THEN fstamp=MAX(0,jfdstmpa);
   ELSE IF 1012 LE qyear LE 1035 THEN fstamp=MAX(0,SUM(OF foodsmpx foodspbx));
   ELSE IF 1041 LE qyear LE 1131 THEN fstamp=MAX(0, foodsmpm);
   ELSE IF qyear GE 1132 THEN fstamp=MAX(0, JFS_AMTM);
IF fstamp=. OR fincbtax=. THEN official_pov=.; 
   ELSE IF (fincbtax-fstamp)<pov_cut THEN official_pov=1;
   ELSE official_pov=0; 

scale=((fam_size-perslt18+.7*perslt18)**.7);

***********************
CONSUMPTION MEASURES
***********************;
* adjusted total expenditures;
cons1=(totexp2)/scale;
* consumption, no gov't housing adj, no HI;
* cons2=(tflow+n_tot_vflow1-vflow-miscpq-misccq)/scale;
* consumption including education spending;
*cons2=(tflow2+n_tot_vflow1-vflow-miscpq-misccq+educapq+educacq)/scale+pc_kink;
* NOW DEFINED AS WELL-MEASURED CONSUMPTION (STARTING IN 2016);
trans_tot = (transpq+transcq+n_tot_vflow1-SUM(OF vehfinpq vehfincq vehicpq vehiccq cartknpq cartkncq cartkupq cartkucq caropq carocq));
cons2=(core_cons+n_tot_vflow1+SUM(OF gasmopq gasmocq)-trans_tot-(tflow2-tflow))/scale;

* main consumption; 
cons3=(tflow2+n_tot_vflow1-vflow-miscpq-misccq)/scale+pc_kink;
* unadjusted total expenditures;
cons4=(totexp)/scale;
* consumption plus MOOP; 
*cons5=(tflow2+n_tot_vflow1-vflow-miscpq-misccq+healthpq+healthcq)/scale+pc_kink;
* CORE CONSUMPTION;
cons5=(core_cons)/scale;
* consumption no HI; 
cons6=(tflow2+n_tot_vflow1-vflow-miscpq-misccq)/scale;
* non-housing consumption; 
cons7=(tflow-SUM(OF rendwepq rendwecq hflow)+n_tot_vflow1-vflow-miscpq-misccq)/scale+pc_kink;
* after tax income;
* cons9=(fincatax+nonincmx-fica)/scale;
* CONS9: NOW DEFINED AS WELL-MEASURED CONSUMPTION LESS FOOD (STARTING IN 2016);
cons9=(core_cons+n_tot_vflow1+SUM(OF gasmopq gasmocq)-trans_tot-(tflow2-tflow)-SUM(OF fdhomepq2 fdhomecq2))/scale;

***********************
INCOME MEASURES
***********************;
* before tax income;
IF fstamp=. OR fincbtax=. THEN cons8=.; 
   ELSE cons8=(fincbtax-fstamp)/scale;


IF fam_type=7 THEN wgt23=finlwt*fam_size; 
   ELSE wgt23=0; 

IF ((totexp2)*4)<pov_cut THEN pov1=finlwt;
   ELSE pov1=0;
IF ((tflow+n_tot_vflow1-vflow+pc_kink*fam_size)*4)<pov_cut THEN pov2=finlwt; 
   ELSE pov2=0;    
IF ((tflow2+n_tot_vflow1-vflow+pc_kink*fam_size)*4)<pov_cut THEN pov3=finlwt;
   ELSE pov3=0; 
gap1=(max(0,pov_cut-((MAX(0,totexp2))*4)))*finlwt;
gap2=(max(0,pov_cut-(MAX(0,tflow+n_tot_vflow1-vflow+pc_kink*fam_size))*4))*finlwt;
gap3=(max(0,pov_cut-(MAX(0,tflow2+n_tot_vflow1-vflow+pc_kink*fam_size))*4))*finlwt;

RUN;

****************************************************************
STACK 60-61 72-73 AND 80-PRESENT, MERGE IN PRICES, MEDIAN, THRESHOLDS 
****************************************************************;
DATA poverty;
  SET alldat.famstep6061 pov7273 poverty;

pers1864=fam_size-perslt18-persot64;
wgt20=finlwt*fam_size;
wgt21=finlwt*perslt18; 
wgt22=finlwt*persot64; 
wgt23=finlwt*pers1864;


*****************************************************************************
CREATE MUTUALLY EXCLUSIVE DEMOGRAPHIC GROUPS
USE MARRIED INSTEAD OF MARITAL1 IN 1972-73, BECAUSE ONE OBS HAD MARITAL1=0 IN
THAT SURVEY
*****************************************************************************;
IF 1959 LE ref_year LE 1973 THEN DO;
   IF age_ref LE 64 AND perslt18 GT 0 AND married=0 THEN ftype=1;
     ELSE IF age_ref LE 64 AND perslt18 GT 0 AND married=1 THEN ftype=2;
     ELSE IF age_ref LE 64 AND perslt18=0 AND married=0 THEN ftype=3;
     ELSE IF age_ref LE 64 AND perslt18=0 AND married=1 THEN ftype=4;
     ELSE IF age_ref GE 65 THEN ftype=5; 
END;
IF ref_year GE 1979 THEN DO;
   IF age_ref LE 64 AND perslt18 GT 0 AND marital1 GT 1 THEN ftype=1;
     ELSE IF age_ref LE 64 AND perslt18 GT 0 AND marital1=1 THEN ftype=2;
     ELSE IF age_ref LE 64 AND perslt18=0 AND marital1 GT 1 THEN ftype=3;
     ELSE IF age_ref LE 64 AND perslt18=0 AND marital1=1 THEN ftype=4;
     ELSE IF age_ref GE 65 THEN ftype=5; 
END;


*****************************************************************************
DEFINE CONSISTENT OCCUPATION GROUPS ACROSS YEARS
*****************************************************************************;
IF 801 LE qyear LE 935 THEN DO;
    IF occucod1=1 THEN occ_recode=1 ;
        ELSE IF occucod1=2 THEN occ_recode=2;
        ELSE IF occucod1=3 THEN occ_recode=3;
        ELSE IF occucod1=4 THEN occ_recode=4;
        ELSE IF occucod1=5 THEN occ_recode=5;
        ELSE IF occucod1=6 THEN occ_recode=6;
        ELSE IF occucod1=7 THEN occ_recode=7;
        ELSE IF occucod1 IN(8, 9, 10, 11, .) THEN occ_recode=8;
END;

IF qyear GE 941 THEN DO;
    IF occucod1 IN(1, 2, 3) THEN occ_recode=1;
        ELSE IF occucod1 IN(4, 5, 6, 7) THEN occ_recode=2;
        ELSE IF occucod1 IN(8, 9, 10) THEN occ_recode=3;
        ELSE IF occucod1 IN(11, 12, 13) THEN occ_recode=6;
        ELSE IF occucod1 IN(14, 15) THEN occ_recode=5;
        ELSE IF occucod1 IN(16, 17) THEN occ_recode=4;
        ELSE IF occucod1=18 THEN occ_recode=7;
        ELSE IF occucod1=. THEN occ_recode=8;
END;



RUN;

*********************************************************************************
MERGE IN PRICE DATA
*********************************************************************************;
PROC SORT DATA=poverty;
   BY ref_year;
RUN;
DATA poverty;
MERGE poverty(IN=in1) alldat.prices1980(RENAME=(year=ref_year));
 BY ref_year;
 IF in1;
RUN;


*********************************************************************************
*********************************************************************************
*********************************************************************************
*********************************************************************************
*********************************************************************************
*********************************************************************************;


* CREATE PERMANENT SAS DATASET TO BE USED BY OTHER PROGRAMS;


PROC SORT DATA=alldat.famstep8081;
   BY newid;
RUN;
PROC SORT DATA=alldat.mortgage8817;
   BY newid;
RUN;
PROC SORT DATA=alldat.home_eq9117;
   BY newid;
RUN;
PROC SORT DATA=alldat.heloc9117;
   BY newid;
RUN;
PROC SORT DATA=alldat.home_val8417;
   BY newid;
RUN;
PROC SORT DATA=poverty;
   BY newid;
RUN;


DATA alldat.cons_final60_17;
   MERGE poverty(IN=in1)                     
   			 alldat.famstep8081(KEEP=newid totpval)
   			 alldat.mortgage8817(KEEP=newid principal)
   			 alldat.home_eq9117(KEEP=newid prin_hel)
   			 alldat.heloc9117(KEEP=newid prin_loc)
   			 alldat.home_val8417(KEEP=newid n_hous totpval);
   BY newid;
   IF in1;
   
   ARRAY recode principal prin_hel prin_loc totpval houseval houseval2 n_hous;
   DO OVER recode;
      IF recode=. THEN recode=0;
   END;   
 
      * NOTE FOR 60-81 THIS IS JUST THE VALUE OF THE SAMPLE UNIT, NOT ADDITIONAL HOMES;
   IF 60 LE srv_year LE 62 THEN all_prop_val=houseval2;
      ELSE IF 70 LE srv_year LE 74 THEN all_prop_val=houseval;
      ELSE IF srv_year GE 80 THEN all_prop_val=totpval;
 
RUN;


PROC SORT DATA=alldat.cons_final60_17;
BY newid;
RUN;
PROC SORT DATA=alldat.cons_final60_16;
BY newid;
RUN;

PROC COMPARE BASE=alldat.cons_final60_17
	     COMPARE=alldat.cons_final60_16;
   WHERE qyear LE 1164;
RUN;

ENDSAS;

